#! /bin/bash
###############################################
##
##  Copyright (2020, ) Institute of Software
##      Chinese Academy of Sciences
##          wuheng@iscas.ac.cn
##
###############################################


VERSION="2.0"

###############################################
##
##  System Config, you can modify it
##
###############################################

########      You can modify them      ########

####   Container  #### 
kube_pod_subnet="10.244.0.0/16"
kube_version="1.22.3"
kube_image_server="registry.cn-hangzhou.aliyuncs.com/google_containers"
#kube_runtime="containerd"                # containerd or docker
#kube_runtime_version="1.4.9"             # containerd 
kube_runtime="docker"                     # containerd or docker
kube_runtime_version="20.10.8"            # docker
kube_fip="39.100.71.73"                               # if you deploy Kubernetes on a public cloud, the fip is public IP
kube_software_repo="https://gitee.com/syswu/yamls"  
kube_cni="calico"                         # calico or kubeovn
kube_addon="addon"                        # loki, prometheus, grafana
kube_cicd="cicd"                          # gitlab
kube_crds="crds"                          # crds

#### VirtualMachine ####
kube_virt_type="kvm"                      # kvm or kata
kube_virt_version="train"                


########    Advanced setting       ########
centos_docker_ce_url="https://download.docker.com/linux/centos/\$releasever/\$basearch/stable"
centos_docker_ce_gpg="https://download.docker.com/linux/centos/gpg"
centos_kuberentes_url="https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-\$basearch/"
centos_kuberentes_gpg="https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg"

ubuntu_docker_ce_url="https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/"
ubuntu_docker_ce_gpg="https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/gpg"
ubuntu_kuberentes_url="https://mirrors.aliyun.com/kubernetes/apt/"
ubuntu_kuberentes_gpg="https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg"


########    Do not modify them       ########

# Deprecated

## version
kube_backend_version="2.0.4"
kube_frontend_version="2.0.4"

###############################################
##
## Do not modify the below codes youself
## Create a issue if you need new features,
## or find a bug
##
###############################################
function get-token
{
  echo $(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kubernetes-client | awk '{print $1}') | grep "token:" | awk -F":" '{print$2}' | sed 's/ //g')
}

function get-arch()
{
  if [[ $(arch) == "x86_64" ]]
  then
    echo "amd64"
  elif [[ $(arch) == "aarch64" ]]
  then
    echo "arm64"
  else
    echo "only support x86_64(amd64) and aarch64(arm64)"
    exit 1
  fi
}

function get-ip()
{
  name=$(hostname | tr '[A-Z]' '[a-z]')
  echo $(kubectl get no $name -o yaml | grep "\- address:" | head -1 | awk -F":" '{print$2}' | sed  's/^[ \t]*//g')
}

function stop-service()
{
  for ser in "$@"
  do
    systemctl stop $ser
    systemctl disable $ser
  done 
}

function start-service()
{
  for ser in "$@"
  do
    systemctl start $ser
    systemctl enable $ser
  done 
}

function replace()
{
   sed -ie "s/$1/$2/g" $3
}

###########################################################
##
##  init-env
##
##########################################################

function init-env()
{
  install=""
  osname=""
  if [[ -n $(cat /etc/os-release | grep centos) ]]
  then
    install="yum"
    osname="centos"
  elif [[ -n $(cat /etc/os-release | grep ubuntu) ]]
  then
    install="apt"
    osname="ubuntu"
  else
    echo "only support centos and ubuntu."
    exit 1
  fi
  disable-$osname-security
  add-$install-repository
  $install-$kube_runtime-k8s
  init-kubeadm-kubeconfig

  echo 2048 > /proc/sys/net/core/somaxconn && echo never > /sys/kernel/mm/transparent_hugepage/enabled
  #install-helm 
}



function disable-centos-security()
{
  sed -ie 's/SELINUX=permissive/SELINUX=disabled/g' /etc/selinux/config
  sed -ie 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
  stop-service firewalld
}

function disable-ubuntu-security()
{
  sudo ufw disable
}



function add-yum-repository()
{
  echo -e "[docker-ce-stable]\nname=Docker CE Stable - \$basearch \nbaseurl=$centos_docker_ce_url\nenabled=1 \ngpgcheck=1 \ngpgkey=$centos_docker_ce_gpg" > /etc/yum.repos.d/docker-ce.repo
  echo -e "[kubernetes] \nname=Kubernetes - \$basearch \nbaseurl=$centos_kuberentes_url \nenabled=1 \ngpgcheck=0 \nrepo_gpgcheck=0 \ngpgkey=$centos_kuberentes_gpg" > /etc/yum.repos.d/kubernetes.repo

}

function add-apt-repository()
{
  arch=`get-arch`

  apt-get install apt-transport-https ca-certificates curl gnupg lsb-release lrzsz -y

  rm -rf /usr/share/keyrings/docker-archive-keyring.gpg
  curl -fsSL $ubuntu_docker_ce_gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
  echo  "deb [arch=$arch signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] $ubuntu_docker_ce_url $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

  rm -rf /usr/share/keyrings/kubernetes-archive-keyring.gpg
  curl -fsSL $ubuntu_kuberentes_gpg | sudo gpg --dearmor -o /usr/share/keyrings/kubernetes-archive-keyring.gpg
  echo  "deb [arch=$arch signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] $ubuntu_kuberentes_url kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list > /dev/null

}



function yum-docker-k8s()
{
  yum install -y wget docker-ce-${kube_runtime_version} kubectl-${kube_version} kubelet-${kube_version} kubeadm-${kube_version}
  config-docker 
}


function apt-docker-k8s()
{
  dVer=$(apt-cache madison docker-ce | grep $kube_runtime_version | awk -F"|" '{print$2}' | sed 's/ //g')
  kVer=$(apt-cache madison kubelet | grep $kube_version | awk -F"|" '{print$2}' | sed 's/ //g')
  apt-get update
  apt-get install docker-ce=$dVer kubectl=$kVer kubelet=$kVer kubeadm=$kVer -y
  
  config-docker
}

function yum-containerd-k8s()
{
  yum install -y wget containerd.io-${kube_runtime_version} kubectl-${kube_version} kubelet-${kube_version} kubeadm-${kube_version}
  
  config-containerd
}

function apt-containerd-k8s()
{

  kVer=$(apt-cache madison kubelet | grep $kube_version | awk -F"|" '{print$2}' | sed 's/ //g')
  apt-get update
  apt-get install containerd.io kubectl=$kVer kubelet=$kVer kubeadm=$kVer -y
  
  config-containerd
}



function config-docker()
{
  ## kubernetes >= 1.22 only supports systemd
  mkdir -p /etc/docker
  echo "{" > /etc/docker/daemon.json
  echo "  \"exec-opts\": [\"native.cgroupdriver=systemd\"]" >> /etc/docker/daemon.json
  echo "}" >> /etc/docker/daemon.json
  systemctl daemon-reload

  start-service docker kubelet
}

function config-containerd()
{
  echo "overlay" > /etc/modules-load.d/containerd.conf
  echo "br_netfilter" >> /etc/modules-load.d/containerd.conf
  
  modprobe overlay
  modprobe br_netfilter

  echo "net.bridge.bridge-nf-call-iptables  = 1" > /etc/sysctl.d/99-kubernetes-cri.conf
  echo "net.ipv4.ip_forward                 = 1" >> /etc/sysctl.d/99-kubernetes-cri.conf
  echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.d/99-kubernetes-cri.conf

  sysctl --system

  echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet

  containerd config default | sudo tee /etc/containerd/config.toml

  def_repo="k8s.gcr.io"
  real_repo=${kube_image_server/\//\\\/}
  old_version=$(cat /etc/containerd/config.toml | grep "k8s.gcr.io" | awk -F"/" '{print$2}' | awk -F"\"" '{print$1}')
  new_version=$(kubeadm config images list | grep pause | awk -F"/" '{print$2}' | awk -F"\"" '{print$1}')
  sed -ie "s/$def_repo/$real_repo/g" /etc/containerd/config.toml
  sed -ie "s/$old_version/$new_version/g" /etc/containerd/config.toml 

  start-service containerd kubelet
}



function init-kubeadm-kubeconfig()
{
  if [[ ! -d "/etc/kubernetes" ]]
  then
    mkdir /etc/kubernetes
  fi
  echo -e "apiVersion: kubeadm.k8s.io/v1beta2" > /etc/kubernetes/kubeadm.yaml
  echo -e "kind: InitConfiguration" >> /etc/kubernetes/kubeadm.yaml
  echo -e "bootstrapTokens:" >> /etc/kubernetes/kubeadm.yaml
  echo -e "  - ttl: \"0\"" >> /etc/kubernetes/kubeadm.yaml
  if [[ $kube_runtime = "containerd" ]]
  then
    echo -e "nodeRegistration:" >> /etc/kubernetes/kubeadm.yaml
    echo -e "  criSocket: /run/containerd/containerd.sock" >> /etc/kubernetes/kubeadm.yaml
  fi
  echo -e "---" >> /etc/kubernetes/kubeadm.yaml
  echo -e "apiVersion: kubeadm.k8s.io/v1beta2" >> /etc/kubernetes/kubeadm.yaml
  echo -e "kind: ClusterConfiguration" >> /etc/kubernetes/kubeadm.yaml
  if [[ -n $kube_fip ]]
  then
  echo -e "controlPlaneEndpoint: \"$kube_fip:6443\"" >> /etc/kubernetes/kubeadm.yaml
  fi
  echo -e "networking:" >> /etc/kubernetes/kubeadm.yaml
  echo -e "  podSubnet: \"${kube_pod_subnet}\"" >> /etc/kubernetes/kubeadm.yaml
  echo -e "kubernetesVersion: \"v${kube_version}\"" >> /etc/kubernetes/kubeadm.yaml
  echo -e "imageRepository: \"${kube_image_server}\"" >> /etc/kubernetes/kubeadm.yaml
}



function install-helm()
{
  helm plugin install git@github.com:chartmuseum/helm-push.git
}


###########################################################
##
##  init-kube
##
##########################################################

function init-kube()
{
  # [optional] just consider performance
  swapoff -a      
  
  # [required] prerequires
  systemctl restart kubelet
  echo "1" > /proc/sys/net/bridge/bridge-nf-call-iptables
  echo "1" > /proc/sys/net/ipv4/ip_forward

  # [required], see https://kubernetes.io/docs/reference/config-api/kubeadm-config.v1beta2/
  kubeadm init --config /etc/kubernetes/kubeadm.yaml
  rm -rf $HOME/.kube
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
  iptables -P FORWARD ACCEPT
}


###########################################################
##
##  init-cni
##
##########################################################

function init-cni()
{
  init-cni-$kube_cni
}


function init-cni-calico()
{
  kubectl apply -f $kube_software_repo/raw/master/cni/$kube_cni.yaml
}


function init-cni-kubeovn()
{
  curl $kube_software_repo/raw/master/cni/$kube_cni.sh | bash -
}

###########################################################
##
##  init-addon
##
##########################################################

function init-addon()
{
  rm -rf /tmp/list
  wget -P /tmp/ $kube_software_repo/raw/master/$kube_addon/list
  for line in `cat /tmp/list`
  do
    kubectl apply -f $line
  done
}

###########################################################
##
##  init-cicd
##
##########################################################

function init-cicd()
{
  rm -rf /tmp/list
  wget -P /tmp/ $kube_software_repo/raw/master/$kube_addon/list
  for line in `cat /tmp/list`
  do
    kubectl apply -f $line
  done
}

###########################################################
##
##  init-kvm
##
##########################################################

function init-virt()
{
  init-$kube_virt_type
}

function init-kvm()
{
  install=""
  osname=""
  if [[ -n $(cat /etc/os-release | grep centos) ]]
  then
    install="yum"
    osname="centos"
  elif [[ -n $(cat /etc/os-release | grep ubuntu) ]]
  then
    install="apt"
    osname="ubuntu"
  else
    echo "only support centos and ubuntu."
    exit 1
  fi
  disable-$osname-security
  add-kvm-$install-repository
  $install-install
  start-service libvirtd 
}

function apt-install()
{
  #apt-get install qemu-kvm libvirt-dev libvirt-daemon libvirt-clients -y
  apt-get install qemu-kvm libvirt-bin -y
}

function yum-install()
{
  yum instal qemu-kvm libvirt -y
}

function add-kvm-yum-repository()
{
  yum install centos-release-openstack-$kube_virt_version -y
}

function add-kvm-apt-repository
{
  add-apt-repository cloud-archive:$kube_virt_version
}


###########################################################
##
##  kube-backend
##
##########################################################

yamlUrl="https://gitee.com/syswu/yamls/raw/master/"
plugins="collector/list"

function init-k8s-plugins()
{
   rm -rf /etc/kubernetes/kubeenv.list
   array=(${plugins//,/ })
   for var in ${array[@]}
   do
     curl $yamlUrl$var > /tmp/list
     for line in `cat /tmp/list`
     do
       echo ${line} >> /etc/kubernetes/kubeenv.list
     done
   done
}

###########################################################
##
##  kube-backend
##
##########################################################

backendUrl="https://gitee.com/syswu/yamls/raw/master/backend"
tokenFile="kube-token.yaml"
databaseFile="kube-database-postgres.yaml"
messageFile="kube-message-rabbitmq.yaml"
mirrorFile="kube-runtime-mirror.yaml"
mapperFile="kube-api-mapper.yaml"
dir="/opt/yamls"
type="postgres"

function config-dir()
{
   rm -rf $1
   mkdir -p $1
   chmod 777 $1
}

function apply-yaml()
{
  rm -rf $1/$2
  wget -P $1/ $3/$2
  kubectl apply -f $1/$2
}


function init-crds()
{
  rm -rf /tmp/list
  wget -P /tmp/ $kube_software_repo/raw/master/$kube_crds/list
  for line in `cat /tmp/list`
  do
    kubectl apply -f $line
  done

  kubectl delete clusterrole admin

  rm -rf /tmp/list
  wget -P /tmp/ $frontendUrl/list

  for name in `cat /tmp/list`
  do
    kubectl apply -f $name
  done

}

function init-backend()
{

  config-dir $dir
  config-dir /var/lib/doslab/yamls
 
  apply-yaml $dir $tokenFile    $backendUrl
  apply-yaml $dir $databaseFile $backendUrl
  apply-yaml $dir $mirrorFile   $backendUrl
  apply-yaml $dir $mapperFile   $backendUrl

  token=`get-token`

  # host and url
  host=`get-ip`
  url="https:\/\/"$host":6443"

  #@ port
  dbport="30306"

  sed -i "s/#type#/$type/g" $dir/$mirrorFile
  sed -i "s/#host#/$host/g" $dir/$mirrorFile
  sed -i "s/#port#/$dbport/g" $dir/$mirrorFile
  sed -i "s/#url#/$url/g" $dir/$mirrorFile
  sed -i "s/#token#/$token/g" $dir/$mirrorFile
  sed -i "s/#version#/$kube_backend_version/g" $dir/$mirrorFile

  sed -i "s/#type#/$type/g" $dir/$mapperFile
  sed -i "s/#host#/$host/g" $dir/$mapperFile
  sed -i "s/#port#/$dbport/g" $dir/$mapperFile
  sed -i "s/#url#/$url/g" $dir/$mapperFile
  sed -i "s/#token#/$token/g" $dir/$mapperFile
  sed -i "s/#version#/$kube_backend_version/g" $dir/$mapperFile

  while true
  do
    res=$(kubectl get po -A | grep kube-database |  grep -v Running | grep -v NAMESPACE)
    if [[ -z "$res" ]]
    then
      break
    fi
    sleep 5s
  done

  kubectl apply -f $dir/$mirrorFile
  kubectl apply -f $dir/$mapperFile
 

  while true
  do
    res=$(kubectl get po -A | grep kube-api-mapper |  grep -v Running | grep -v NAMESPACE)
    if [[ -z "$res" ]]
    then
      break
    fi
    sleep 5s
  done

  if [[ -n $kube_fip ]]
  then
    host=$kube_fip
  fi

   
  curl -H "Content-Type: application/json" -X POST -d '{"json": {"name": "admin","rules": [{"apiGroups": ["*"],"resources": ["*"],"verbs": ["*"]}]}}' "http://$host:30308/kubesys/user/createUserRole"
}

function reset-backend()
{
  kubectl delete deploy kube-runtime-mirror -n kube-system
  kubectl delete deploy kube-api-mapper -n kube-system
  kubectl delete svc kube-api-mapper -n kube-system
}

###########################################################
##
##  kube-frontend
##
##########################################################

frontendUrl="https://gitee.com/syswu/yamls/raw/master/frontend"
config_dir="/var/lib/doslab/frontend"
config_name="config.js"
frontendFile="kube-frontend.yaml"

function init-frontend()
{

  # make and config config.js 
  rm -rf $config_dir
  mkdir -p $config_dir
  echo "window.g = {" > $config_dir/$config_name
  if [[ -n $kube_fip ]]
  then
    echo "    baseURL: 'http://$kube_fip:30308/kubesys'" >> $config_dir/$config_name
  else
    echo "    baseURL: 'http://`get-ip`:30308/kubesys'" >> $config_dir/$config_name
  fi
  echo "}" >> $config_dir/$config_name
  
  # make and config yaml
  rm -rf /$dir/$frontendFile 
  wget -P $dir/ $frontendUrl/$frontendFile
  sed -i "s/#version#/$kube_frontend_version/g" $dir/$frontendFile

  # apply yaml
  kubectl apply -f /$dir/$frontendFile
}


function reset-frontend()
{
  kubectl delete deploy kube-frontend -n kube-system
  kubectl delete svc kube-frontend -n kube-system
}


###########################################################
##
##  help
##
##########################################################


function cmddesc()
{
  echo -e "Welcome to kubeinst ($VERSION), install Kubernetes-based systems from scratch.\n"
}


function help()
{
  cmddesc
  echo -e "Commands:"
  echo -e "  init-env             :\t(Init): install conatiner/docker, kubeadm, kubelet and kubectl"
  echo -e "  init-kube            :\t(Init): initialize kubernetes cluster"
  echo -e "  init-cni             :\t(Init): deploy calico or kubeovn, see https://gitee.com/syswu/yamls/cni"
  echo -e "  init-addon           :\t(Init): deploy loki, prometheus, grafana and superset, see https://gitee.com/syswu/yamls/addon"
  echo -e "  init-gpu             :\t(Init): deploy GPU manager, see https://github.com/kubesys/kube-gpu"
  echo -e "  init-virt            :\t(Init): deploy kvm, libvirt and vmm, see https://github.com/kubesys/kube-vmm"
#  echo -e "  init-crds            :\t(Init): deploy crds, see https://gitee.com/syswu/yamls/crds"
#  echo -e "  init-backend         :\t(Init): deploy backend services, see https://gitee.com/syswu/yamls/backend"
#  echo -e "  init-frontend        :\t(Init): deploy dashboard, see https://gitee.com/syswu/yamls/frontend"
}


case $1 in
  "init-env")
    init-env $*
    ;;
  "init-kube")
    init-kube $*
    ;;
  "init-cni")
    init-cni $*
    ;;
  "init-addon")
    init-addon $*
    ;;
  "init-cicd")
    init-cicd $*
    ;;
  "init-gpu")
    init-gpu $*
    ;;
  "init-virt")
    init-virt $*
    ;;
  "init-crds")
    init-crds $*
    ;;
  "init-backend")
    init-backend $*
    ;;
  "init-frontend")
    init-frontend $*
    ;;
  "init-kvm-env")
    init-kvm-env $*
    ;;
  "reset-kube")
    reset-kube $*
    ;;
  "reset-backend")
    reset-backend $*
    ;;
  "reset-frontend")
    reset-frontend $*
    ;;
  "--help")
    help
    ;;
  *)
  help
  ;;
esac
